{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `completeness_chart`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": [
     "hide_input"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "  #altair-viz-cba7a0fa33c74db39285b0e43511375e.vega-embed {\n",
       "    width: 100%;\n",
       "    display: flex;\n",
       "  }\n",
       "\n",
       "  #altair-viz-cba7a0fa33c74db39285b0e43511375e.vega-embed details,\n",
       "  #altair-viz-cba7a0fa33c74db39285b0e43511375e.vega-embed details summary {\n",
       "    position: relative;\n",
       "  }\n",
       "</style>\n",
       "<div id=\"altair-viz-cba7a0fa33c74db39285b0e43511375e\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-cba7a0fa33c74db39285b0e43511375e\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-cba7a0fa33c74db39285b0e43511375e\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.17.0?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function maybeLoadScript(lib, version) {\n",
       "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
       "      return (VEGA_DEBUG[key] == version) ?\n",
       "        Promise.resolve(paths[lib]) :\n",
       "        new Promise(function(resolve, reject) {\n",
       "          var s = document.createElement('script');\n",
       "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "          s.async = true;\n",
       "          s.onload = () => {\n",
       "            VEGA_DEBUG[key] = version;\n",
       "            return resolve(paths[lib]);\n",
       "          };\n",
       "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "          s.src = paths[lib];\n",
       "        });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else {\n",
       "      maybeLoadScript(\"vega\", \"5\")\n",
       "        .then(() => maybeLoadScript(\"vega-lite\", \"5.17.0\"))\n",
       "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"layer\": [{\"mark\": \"rect\", \"encoding\": {\"color\": {\"field\": \"completeness\", \"legend\": null, \"scale\": {\"scheme\": \"darkred\", \"zero\": true}, \"type\": \"quantitative\"}, \"tooltip\": [{\"field\": \"source_dataset\", \"title\": \"Source dataset\", \"type\": \"nominal\"}, {\"field\": \"total_rows_inc_nulls\", \"format\": \",\", \"title\": \"# of records\", \"type\": \"quantitative\"}, {\"field\": \"column_name\", \"title\": \"Column name\", \"type\": \"nominal\"}, {\"field\": \"total_null_rows\", \"format\": \",\", \"title\": \"# of nulls\", \"type\": \"quantitative\"}, {\"field\": \"completeness\", \"format\": \".1%\", \"type\": \"quantitative\"}], \"x\": {\"axis\": {\"labelAngle\": 20}, \"field\": \"column_name\", \"sort\": {\"field\": \"mean_comp\", \"order\": \"descending\"}, \"title\": \"Column name\", \"type\": \"nominal\"}, \"y\": {\"field\": \"source_dataset\", \"title\": \"Source dataset\", \"type\": \"nominal\"}}, \"title\": \"Column completeness by source dataset\", \"transform\": [{\"joinaggregate\": [{\"op\": \"mean\", \"field\": \"completeness\", \"as\": \"mean_comp\"}], \"groupby\": [\"column_name\"]}]}, {\"mark\": {\"type\": \"text\"}, \"encoding\": {\"color\": {\"condition\": {\"test\": \"datum['completeness'] < 0.5\", \"value\": \"white\"}, \"value\": \"black\"}, \"text\": {\"field\": \"completeness\", \"format\": \".0%\", \"type\": \"quantitative\"}, \"x\": {\"axis\": {\"labelAngle\": 0}, \"field\": \"column_name\", \"sort\": {\"field\": \"mean_comp\", \"order\": \"descending\"}, \"type\": \"nominal\"}, \"y\": {\"field\": \"source_dataset\", \"type\": \"nominal\"}}, \"transform\": [{\"joinaggregate\": [{\"op\": \"mean\", \"field\": \"completeness\", \"as\": \"mean_comp\"}], \"groupby\": [\"column_name\"]}]}], \"data\": {\"name\": \"data-4d5228724574854f533de3811eb76968\"}, \"height\": {\"step\": 40}, \"width\": {\"step\": 40}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.9.3.json\", \"datasets\": {\"data-4d5228724574854f533de3811eb76968\": [{\"source_dataset\": \"input_data_2\", \"column_name\": \"unique_id\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"unique_id\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"first_name\", \"total_null_rows\": 90, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8199999928474426}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"first_name\", \"total_null_rows\": 79, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8420000076293945}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"surname\", \"total_null_rows\": 100, \"total_rows_inc_nulls\": 500, \"completeness\": 0.800000011920929}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"surname\", \"total_null_rows\": 81, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8379999995231628}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"dob\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"dob\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"city\", \"total_null_rows\": 87, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8259999752044678}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"city\", \"total_null_rows\": 100, \"total_rows_inc_nulls\": 500, \"completeness\": 0.800000011920929}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"email\", \"total_null_rows\": 104, \"total_rows_inc_nulls\": 500, \"completeness\": 0.7919999957084656}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"email\", \"total_null_rows\": 107, \"total_rows_inc_nulls\": 500, \"completeness\": 0.7860000133514404}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"cluster\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"cluster\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.LayerChart(...)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chart"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "!!! info \"At a glance\"\n",
    "    **Useful for:** Looking at which columns are populated across datasets. \n",
    "\n",
    "    **API Documentation:** [completeness_chart()](../api_docs/exploratory.md#splink.exploratory.completeness_chart)\n",
    "\n",
    "    **What is needed to generate the chart?** A `linker` with some data."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What the chart shows\n",
    "\n",
    "The `completeness_chart` shows the proportion of populated (non-null) values in the columns of multiple datasets."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "??? note \"What the chart tooltip shows\"\n",
    "\n",
    "    ![](./img/completeness_chart_tooltip.png)\n",
    "\n",
    "    The tooltip shows a number of values based on the panel that the user is hovering over, including:\n",
    "\n",
    "    - The dataset and column name\n",
    "    - The count and percentage of non-null values in the column for the relelvant dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### How to interpret the chart\n",
    "\n",
    "Each panel represents the percentage of non-null values in a given dataset-column combination. The darker the panel, the lower the percentage of non-null values.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<hr>"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Actions to take as a result of the chart\n",
    "\n",
    "Only choose features that are sufficiently populated across all datasets in a linkage model.\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Worked Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": [
     "hide_output"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "  #altair-viz-5c0874f275664010a54a66fb245d4abc.vega-embed {\n",
       "    width: 100%;\n",
       "    display: flex;\n",
       "  }\n",
       "\n",
       "  #altair-viz-5c0874f275664010a54a66fb245d4abc.vega-embed details,\n",
       "  #altair-viz-5c0874f275664010a54a66fb245d4abc.vega-embed details summary {\n",
       "    position: relative;\n",
       "  }\n",
       "</style>\n",
       "<div id=\"altair-viz-5c0874f275664010a54a66fb245d4abc\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
       "  (function(spec, embedOpt){\n",
       "    let outputDiv = document.currentScript.previousElementSibling;\n",
       "    if (outputDiv.id !== \"altair-viz-5c0874f275664010a54a66fb245d4abc\") {\n",
       "      outputDiv = document.getElementById(\"altair-viz-5c0874f275664010a54a66fb245d4abc\");\n",
       "    }\n",
       "    const paths = {\n",
       "      \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
       "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
       "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.17.0?noext\",\n",
       "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
       "    };\n",
       "\n",
       "    function maybeLoadScript(lib, version) {\n",
       "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
       "      return (VEGA_DEBUG[key] == version) ?\n",
       "        Promise.resolve(paths[lib]) :\n",
       "        new Promise(function(resolve, reject) {\n",
       "          var s = document.createElement('script');\n",
       "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "          s.async = true;\n",
       "          s.onload = () => {\n",
       "            VEGA_DEBUG[key] = version;\n",
       "            return resolve(paths[lib]);\n",
       "          };\n",
       "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
       "          s.src = paths[lib];\n",
       "        });\n",
       "    }\n",
       "\n",
       "    function showError(err) {\n",
       "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
       "      throw err;\n",
       "    }\n",
       "\n",
       "    function displayChart(vegaEmbed) {\n",
       "      vegaEmbed(outputDiv, spec, embedOpt)\n",
       "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
       "    }\n",
       "\n",
       "    if(typeof define === \"function\" && define.amd) {\n",
       "      requirejs.config({paths});\n",
       "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
       "    } else {\n",
       "      maybeLoadScript(\"vega\", \"5\")\n",
       "        .then(() => maybeLoadScript(\"vega-lite\", \"5.17.0\"))\n",
       "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
       "        .catch(showError)\n",
       "        .then(() => displayChart(vegaEmbed));\n",
       "    }\n",
       "  })({\"config\": {\"view\": {\"continuousWidth\": 400, \"continuousHeight\": 300}}, \"layer\": [{\"mark\": \"rect\", \"encoding\": {\"color\": {\"field\": \"completeness\", \"legend\": null, \"scale\": {\"scheme\": \"darkred\", \"zero\": true}, \"type\": \"quantitative\"}, \"tooltip\": [{\"field\": \"source_dataset\", \"title\": \"Source dataset\", \"type\": \"nominal\"}, {\"field\": \"total_rows_inc_nulls\", \"format\": \",\", \"title\": \"# of records\", \"type\": \"quantitative\"}, {\"field\": \"column_name\", \"title\": \"Column name\", \"type\": \"nominal\"}, {\"field\": \"total_null_rows\", \"format\": \",\", \"title\": \"# of nulls\", \"type\": \"quantitative\"}, {\"field\": \"completeness\", \"format\": \".1%\", \"type\": \"quantitative\"}], \"x\": {\"axis\": {\"labelAngle\": 20}, \"field\": \"column_name\", \"sort\": {\"field\": \"mean_comp\", \"order\": \"descending\"}, \"title\": \"Column name\", \"type\": \"nominal\"}, \"y\": {\"field\": \"source_dataset\", \"title\": \"Source dataset\", \"type\": \"nominal\"}}, \"title\": \"Column completeness by source dataset\", \"transform\": [{\"joinaggregate\": [{\"op\": \"mean\", \"field\": \"completeness\", \"as\": \"mean_comp\"}], \"groupby\": [\"column_name\"]}]}, {\"mark\": {\"type\": \"text\"}, \"encoding\": {\"color\": {\"condition\": {\"test\": \"datum['completeness'] < 0.5\", \"value\": \"white\"}, \"value\": \"black\"}, \"text\": {\"field\": \"completeness\", \"format\": \".0%\", \"type\": \"quantitative\"}, \"x\": {\"axis\": {\"labelAngle\": 0}, \"field\": \"column_name\", \"sort\": {\"field\": \"mean_comp\", \"order\": \"descending\"}, \"type\": \"nominal\"}, \"y\": {\"field\": \"source_dataset\", \"type\": \"nominal\"}}, \"transform\": [{\"joinaggregate\": [{\"op\": \"mean\", \"field\": \"completeness\", \"as\": \"mean_comp\"}], \"groupby\": [\"column_name\"]}]}], \"data\": {\"name\": \"data-4d5228724574854f533de3811eb76968\"}, \"height\": {\"step\": 40}, \"width\": {\"step\": 40}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.9.3.json\", \"datasets\": {\"data-4d5228724574854f533de3811eb76968\": [{\"source_dataset\": \"input_data_2\", \"column_name\": \"unique_id\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"unique_id\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"first_name\", \"total_null_rows\": 90, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8199999928474426}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"first_name\", \"total_null_rows\": 79, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8420000076293945}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"surname\", \"total_null_rows\": 100, \"total_rows_inc_nulls\": 500, \"completeness\": 0.800000011920929}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"surname\", \"total_null_rows\": 81, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8379999995231628}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"dob\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"dob\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"city\", \"total_null_rows\": 87, \"total_rows_inc_nulls\": 500, \"completeness\": 0.8259999752044678}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"city\", \"total_null_rows\": 100, \"total_rows_inc_nulls\": 500, \"completeness\": 0.800000011920929}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"email\", \"total_null_rows\": 104, \"total_rows_inc_nulls\": 500, \"completeness\": 0.7919999957084656}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"email\", \"total_null_rows\": 107, \"total_rows_inc_nulls\": 500, \"completeness\": 0.7860000133514404}, {\"source_dataset\": \"input_data_2\", \"column_name\": \"cluster\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}, {\"source_dataset\": \"input_data_1\", \"column_name\": \"cluster\", \"total_null_rows\": 0, \"total_rows_inc_nulls\": 500, \"completeness\": 1.0}]}}, {\"mode\": \"vega-lite\"});\n",
       "</script>"
      ],
      "text/plain": [
       "alt.LayerChart(...)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from splink import splink_datasets, DuckDBAPI\n",
    "from splink.exploratory import completeness_chart\n",
    "\n",
    "df = splink_datasets.fake_1000\n",
    "\n",
    "# Split a simple dataset into two, separate datasets which can be linked together.\n",
    "df_l = df.sample(frac=0.5)\n",
    "df_r = df.drop(df_l.index)\n",
    "\n",
    "\n",
    "chart = completeness_chart([df_l, df_r], db_api=DuckDBAPI())\n",
    "chart"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
